import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.TextUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.utils.HttpConstant;
//为了完成视频，爬虫设计整体思路为先由up主id找到投稿视频列表，由视频列表日期统计某投稿日期时的投稿数
public class BiliPageProcessor implements PageProcessor {

	private Site site = Site.me()
            .setRetryTimes(3)
            .setTimeOut(50000)
            .setSleepTime(0)        //跟据试验，http://space.bilibili.com/ajax/member/GetInfo接口有IP接入限制，估计是60s内上限150次
            .setCycleRetryTimes(3)
            .setUseGzip(true)
            .addHeader("Host","space.bilibili.com")
            .addHeader("Connection", "keep-alive")
            .addHeader("Origin", "https://space.bilibili.com")
            .addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
            .addHeader("Accept","application/json, text/plain, */*")
            .addHeader("Accept-Language","zh-CN,zh;q=0.9")
            .addHeader("Accept-Encoding","gzip, deflate, br")
            .addHeader("X-Requested-With","XMLHttpRequest")
            .addHeader("Content-Type","application/x-www-form-urlencoded")
            .addHeader("Referer","http://space.bilibili.com/1506485/");
            
	public static int startMid = 101;
	public static String vurl = "http://api.bilibili.com/archive_stat/stat?aid=";//单视频详细信息
	public static String uploadUrl = "https://api.bilibili.com/x/space/navnum?mid=";//投稿数相关
	public static String fansListUrl = "https://api.bilibili.com/x/relation/followings?vmid=1506485&pn=1";//粉丝列表
	public static String relationUrl = "https://api.bilibili.com/x/relation/stat?vmid=63231";//粉丝与关注
	public static String vListUrl = "https://space.bilibili.com/ajax/member/getSubmitVideos?mid=";//投稿列表视频详细信息
	public static File f = new File("E:\\log4j\\webmagic\\bili.csv");
    public static int pagesize = 100;
    public static String lastDateSet = "2018-10";
	
	@Override
	public void process(Page page) {
//		page.putField("aid", page.getHtml().regex("\"aid\":\\d+"));
//		System.out.println(page.getHtml().regex("\"aid\":\\d+"));
//		System.out.println(page.getHtml());
//		System.out.println(page.getRequest());
//		System.out.println(page.getHtml().regex("(author\":\"\\S+?)\"").toString().substring(9));
		int counts = Integer.valueOf(page.getHtml().regex("count\":\\d+,\"pages").regex("\\d+").toString());
		int pages = Integer.valueOf(page.getHtml().regex("count\":\\d+,\"pages\":\\d+").regex("pages\":\\d+").regex("\\d+").toString());
		int p = Integer.valueOf(page.getUrl().regex("page=\\d+").regex("\\d+").toString());
//		System.out.println(decodeUnicode(author));
		if(p <= pages){
		String author = page.getHtml().regex("(author\":\"\\S+?)\"").toString().substring(9);
//		System.out.println(page.getHtml().regex("created\":\\d+").all().size());
		List<String> timeList = page.getHtml().regex("created\":\\d+").all();
//		System.out.println(TimeStamp2Date(timeList.get(timeList.size()-1).substring(9),"yyyy-MM-dd"));
		UserModule u = new UserModule();
		u.setType("投稿数");
		u.setAuthor(decodeUnicode(author));
//		timeList.add(0, "created\":1535731200");
		List<String> dateList = new ArrayList<String>();
		for(int i = 0; i<timeList.size(); i++){
			dateList.add(TimeStamp2Date(timeList.get(i).substring(9),"yyyy-MM"));
		}
//		for(Iterator itr = dateList.iterator(); itr.hasNext();){
//			System.out.println(itr.next().toString());
//		}
		Collections.reverse(dateList);
		int[] ammount = new int[1000];
		for(int i=0; i<1000; i++)ammount[i]=1;
//fucking methor to record the ammounts of different date!
		for(int i=1 ; i<dateList.size(); i++){
			while(i<dateList.size()&&dateList.get(i-1).equals(dateList.get(i))){
				dateList.remove(i);
				ammount[i]++;
			}
		}
		for(int i = 0; i<dateList.size(); i++){
//			System.out.println(dateList.get(i));
			ammount[i+2]=ammount[i+2]+ammount[i+1];
//			System.out.println(ammount[i+1]);
		}
//fucking key methor to fix the request order problem!
		int lastPageAmmount = counts%pagesize;
		for(int i = 0; i<dateList.size(); i++){
//			System.out.println(dateList.get(i));
			if(p!=pages)
			ammount[i+1]+=((pages-p-1)*pagesize+lastPageAmmount);
//			System.out.println(ammount[i+1]);
			u.setAmmount(ammount[i+1]);
			u.setDate(dateList.get(i));
//			writeToCsvFile(f, u);
			while(i<(dateList.size()-1) && !dateList.get(i).equals(dateList.get(i+1))){
				u.setDate(dateList.set(i, incDateByOneTo(dateList.get(i))));
				writeToCsvFile(f, u);
			}
			int incTime = calcDateToDateTimes(dateList.get(i), lastDateSet);
			if(p == 1){
			if(i==(dateList.size()-1)){
//				System.out.println(calcDateToDateTimes(dateList.get(i), lastDateSet));
				for(int times = 0; times<incTime; times++){					
					u.setDate(dateList.set(i, incDateByOneTo(dateList.get(i))));
					writeToCsvFile(f, u);
				}
			}
			}
		}
		

		
		
//错误写法
//		for(int i = 0; i<timeList.size(); i++){			
//			u.setAmmount(counts-i-(p-1)*pagesize);
//			if(i==0){
//			for(int t = 1535893752; t>Integer.valueOf(timeList.get(i).substring(9)); t-=(86400*30)){
//				u.setDate(TimeStamp2Date(String.valueOf(t),"yyyy-MM"));
//				writeToCsvFile(f,u);
//			}
//			}
//			if(i<(timeList.size()-1)){
//			for(int temp = Integer.valueOf(timeList.get(i).substring(9)); temp>Integer.valueOf(timeList.get(i+1).substring(9)); temp-=(86400*30)){
//			u.setDate(TimeStamp2Date(String.valueOf(temp),"yyyy-MM"));
//			writeToCsvFile(f,u);
//			}
//			}
//			else{	
//			u.setDate(TimeStamp2Date(timeList.get(i).substring(9),"yyyy-MM"));
//			writeToCsvFile(f,u);
//			}
//			System.out.println(u.author);
//			System.out.println(u.type);
//			System.out.println(u.ammount);
//			System.out.println(u.date);
//		}
		page.addTargetRequest(new Request(page.getUrl().regex(".*&page=").toString()+(++p)));
		}
		else
		System.out.println("finish "+ page.getUrl().regex("mid=\\d+"));
//		System.out.println(page.getHtml().regex("created\":\\d+").all().size());
		
	}
	
	public static int calcDateToDateTimes(String date1, String date2){
		int count = 2;
		String datet = date1;
		while(!(datet=incDateByOneTo(datet)).equals(date2)){
			count++;
		}
		return count;
	}
	
	public static String incDateByOneTo(String date){
		String newDate = date;
		int temp = Integer.valueOf(newDate.substring(5, 7))+1;
//		System.out.println(temp);
		if(temp>12){
			newDate = (Integer.valueOf(date.substring(0, 4))+1)+"-"+"01";
		}
		else{
			newDate = newDate.substring(0, 4)+"-"+String.format("%02d", temp);
		}		
		return newDate;
	}
	
	public static void writeToCsvFile(File f,UserModule u){
		if(!f.exists()){
			try {
				f.createNewFile();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		else{
			FileWriter w;
			try {
				w = new FileWriter(f,true);
				w.write(u.author+','+u.type+','+u.ammount+','+u.date+"\n");
				w.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}
	
	
	//unicode转中文
	private static String decodeUnicode ( String asciicode )
    {
        String[] asciis = asciicode.split ("\\\\u");
        String nativeValue = asciis[0];
        try
        {
            for ( int i = 1; i < asciis.length; i++ )
            {
                String code = asciis[i];
                nativeValue += (char) Integer.parseInt (code.substring (0, 4), 16);
                if (code.length () > 4)
                {
                    nativeValue += code.substring (4, code.length ());
                }
            }
        }
        catch (NumberFormatException e)
        {
            return asciicode;
        }
        return nativeValue;
    }

	
	public static String TimeStamp2Date(String timestampString, String formats) {//时间戳转换
        if (TextUtils.isEmpty(formats))
            formats = "yyyy-MM-dd";
        Long timestamp = Long.parseLong(timestampString) * 1000;
        String date = new SimpleDateFormat(formats, Locale.CHINA).format(new Date(timestamp));
        return date;
    }

	@Override
	public Site getSite() {
		return site;
	}
	

	public static void main(String[] args) {
//		int mid = 79;
		for(int mid = startMid; mid<=200; mid++)
        Spider.create(new BiliPageProcessor()).addUrl(vListUrl+mid+"&pagesize="+pagesize+"&page=1").thread(10).run();
	}

}
